'''
豆瓣读书 top250:xpath + lxml

需求：
名称 书的信息  评分  评论人数  描述

https://book.douban.com/top250?start=0
https://book.douban.com/top250?start=25
https://book.douban.com/top250?start=50

随机User-Agent,应对网站对同一个User-Agent访问频率的反爬
'''
import requests
import time
import random
from fake_useragent import UserAgent
import pymysql
#from lxml import etree
from lxml import html

'''
create table douban(
title varchar(100),
info varchar(500),
score varchar(100),
commit varchar(100),
comment varchar(500)
)

'''
class Douban:
    def __init__(self):
        self.url = "https://book.douban.com/top250?start={}"
        self.db = pymysql.connect(host='localhost',user='root',password='73327233',db='zjlg',charset='utf8')
        self.cur = self.db.cursor()

    def get_html(self,url):
        '''发送请求并获取响应对象内容的函数'''
        # UserAgent().random:表示随机生成一个User-Agent
        headers = {'User-Agent':UserAgent().random}
        html = requests.get(url=url,headers=headers).content.decode('utf-8','ignore') # 忽略一些不识别的数据
        #print(html)
        self.parse_html(html)

    def parse_html(self,html1):
        '''lxml+xpath 解析提取数据函数'''
        # xpath只能匹配eobj对象中的数据，不能匹配字符串中的数据
        #eobj = etree.HTML(html)
        # from lxml import html
        et = html.etree
        eobj = et.HTML(html1)
        # tablelist = [<element table at ***>,<element table at ***>,<element table at ***>,.......]
        tablelist = eobj.xpath('//table')
        for tab in tablelist:
            # tab = <element table at ***>
            item = {}
            # xpath()返回值是一个列表
            # title_list = ["红楼梦"]
            title_list = tab.xpath('.//div[ @class ="pl2"] /a/@title')
            item['title'] = title_list[0].strip() if title_list else None
            info_list = tab.xpath('.//p[@class="pl"]/text()')
            item['info'] = info_list[0].strip() if info_list else None
            score_list = tab.xpath('.//span[@class="rating_nums"]/text()')
            item['score'] = score_list[0].strip() if score_list else None
            #                                           -1
            #   0  1
            #['(\n               133767人评价         \n)']
            #                     133767人评价
            commit_list = tab.xpath('.//span[@class="pl"]/text()')
            item['commit'] = commit_list[0][1:-1].strip() if commit_list else None
            comment_list = tab.xpath('.//span[@class="inq"]/text()')
            # bool([])的返回值返回值是False
            # 条件表达式，如果if成立则执行comment_list[0].strip()，否则返回None
            item['comment'] = comment_list[0].strip() if comment_list else None
            print(item)
            book = [item['title'],item['info'],item['score'],item['commit'],item['comment']]
            self.save_to_mysql(book)

    def save_to_mysql(self,book):
        '''将数据保存到mysql数据库'''
        ins = 'insert into douban values(%s,%s,%s,%s,%s)'
        self.cur.execute(ins,book)
        self.db.commit()

    def crawl(self):
        for page in range(10):
            page_url = self.url.format(page*25)
            self.get_html(url=page_url)
            # 控制频率
            time.sleep(random.randint(1,3))
        self.cur.close()
        self.db.close()

if __name__ == '__main__':
    douban = Douban()
    douban.crawl()